{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "把 bert 预测的结果 与 大模型原始预测的结果 合并",
   "id": "7517cd2def38ad33"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-14T08:59:33.927621Z",
     "start_time": "2024-09-14T08:59:33.923158Z"
    }
   },
   "cell_type": "code",
   "source": "import os",
   "id": "146f7c5ba46abc84",
   "outputs": [],
   "execution_count": 45
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-14T08:35:24.752050Z",
     "start_time": "2024-09-14T08:35:24.745739Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from datasets import load_from_disk, load_dataset, concatenate_datasets\n",
    "\n",
    "from setting import StaticValues"
   ],
   "id": "c4c1f5c8d4f01bae",
   "outputs": [],
   "execution_count": 19
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-14T08:29:31.970913Z",
     "start_time": "2024-09-14T08:29:31.963303Z"
    }
   },
   "cell_type": "code",
   "source": "name = \"biomedical\"",
   "id": "82e7a662e990baae",
   "outputs": [],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-14T09:08:21.035967Z",
     "start_time": "2024-09-14T09:08:21.029785Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def export_human_csv(name):\n",
    "    sv = StaticValues(name=name)\n",
    "    llm_infer_dataset = load_from_disk(\"/home/jie/gitee/pku_industry/general/output/biomedical/bert_multi_train\")[\n",
    "        \"train\"]\n",
    "\n",
    "    bert_pred_dataset = load_dataset(\"csv\",\n",
    "                                     data_files=os.path.join(sv.home_folder, 'bert_multi_pred.csv'),\n",
    "                                     split='train')\n",
    "\n",
    "    def trans_to_human(name):\n",
    "\n",
    "        \"\"\"\n",
    "        把 bert 多分类的结果 转化为 人类可读的结果\n",
    "        \"\"\"\n",
    "        sv = StaticValues(name=name)\n",
    "        LABEL_NAME = sv.LABEL_NAME\n",
    "\n",
    "        def func(item):\n",
    "            for idx, label_name in enumerate(LABEL_NAME):\n",
    "                item[label_name] = 0\n",
    "                if idx == item['pred_label']:\n",
    "                    item[label_name] = 1\n",
    "            return item\n",
    "\n",
    "        return func\n",
    "\n",
    "    bert_pred_dataset = bert_pred_dataset.map(\n",
    "        trans_to_human(name=name),\n",
    "        remove_columns=['reason',\n",
    "                        'label',\n",
    "                        'industry_info',\n",
    "                        'input_ids',\n",
    "                        'token_type_ids',\n",
    "                        'attention_mask',\n",
    "                        ]\n",
    "    )\n",
    "\n",
    "    new_dataset = concatenate_datasets([llm_infer_dataset, bert_pred_dataset])\n",
    "    new_df = new_dataset.to_pandas()\n",
    "    kw_df = pd.read_csv(sv.KW_CSV)\n",
    "\n",
    "    df = pd.merge(new_df, kw_df, on=['企业名称',\n",
    "                                     '经营范围',\n",
    "                                     '一级行业分类',\n",
    "                                     '二级行业分类',\n",
    "                                     '三级行业分类'], how='inner')\n",
    "    df = df[list(kw_df.columns) + sv.LABEL_NAME + [\"pred_label\"]]\n",
    "    df.to_csv(os.path.join(sv.home_folder, f'{sv.chinese_name}.csv'), index=False)"
   ],
   "id": "9c16cbd4335c6f32",
   "outputs": [],
   "execution_count": 46
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "c74a7be85aa11fdd"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-14T09:09:03.242338Z",
     "start_time": "2024-09-14T09:08:31.907395Z"
    }
   },
   "cell_type": "code",
   "source": "export_human_csv(name)",
   "id": "fe7d25522dcd058f",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Map:   0%|          | 0/319438 [00:00<?, ? examples/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "9a6e8d56fc9240868f40bd9a8fcbec0a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_95447/2646314867.py:40: DtypeWarning: Columns (1,10,11,16,18,23,25,26,27,28,31) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  kw_df = pd.read_csv(sv.KW_CSV)\n"
     ]
    }
   ],
   "execution_count": 47
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## merge",
   "id": "a0557dd81a5810d2"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "merge kw_csv   bert_multi_pred.csv ",
   "id": "ed05c47c3c04af73"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-14T09:09:45.611800Z",
     "start_time": "2024-09-14T09:09:45.353282Z"
    }
   },
   "cell_type": "code",
   "source": "!ls /home/jie/gitee/pku_industry/general/output/biomedical",
   "id": "72513f9be320c9f5",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert\t\t      bert_multi_train\t\t     industry_dataset\r\n",
      "bert_binary_pred.csv  binary_biomedical.pkl\t     log.txt\r\n",
      "bert_binary_train     biomedical_kw.csv\t\t     test.ipynb\r\n",
      "bert_multi_pred.csv   industry_biomedical_multi.pkl  生物医药.csv\r\n"
     ]
    }
   ],
   "execution_count": 48
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "e8933bdcb8df36f6"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-14T09:23:12.472140Z",
     "start_time": "2024-09-14T09:23:12.463152Z"
    }
   },
   "cell_type": "code",
   "source": "150 * 60 * 40 ",
   "id": "8145d8a589777eb5",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "360000"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 49
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "11665c727a34702e"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
